library(ggplot2)
library(dplyr)
library(tidyr)
library("ggpubr")
library(LDATS)
library(ggVennDiagram)
library(stringr)
library(abind)
library(patchwork)

source("utils.R")

Visualization on networks trained in 30 replications on CIFAR 10 training set with 500 sample validation set extracted randomly in each replication.

base_dir <- "../data/data_train_val_c10"
repls <- 0:29
classes <- 10

nets_outputs <- load_network_outputs(base_dir, repls)
ens_outputs <- load_ensemble_outputs(base_dir, repls)
sort_ind <- function(lst)
{
  return(sort(lst, index.return=TRUE, decreasing=TRUE)$ix)
}
nets_test_top_indices <- apply(X=nets_outputs$test_outputs, MARGIN=c(1, 2, 3), FUN=sort_ind)[1, , , ]
ens_tt_top_indices <- apply(X=ens_outputs$train_training, MARGIN=c(1, 2, 3), FUN=sort_ind)[1, , , ]
ens_vt_top_indices <- apply(X=ens_outputs$val_training, MARGIN=c(1, 2, 3), FUN=sort_ind)[1, , , ]
r_n <- length(repls)
samples_n <- dim(nets_outputs$test_labels)[2]
nets_n <- length(nets_outputs$networks)
ens_n <- length(ens_outputs$methods)
test_labs <- nets_outputs$test_labels + 1
dim(test_labs) <- c(r_n, 1, samples_n)
test_labs_nets <- aperm(abind(array(rep(aperm(test_labs, perm=c(2, 1, 3)), nets_n), c(r_n, samples_n, nets_n)), along=3), perm=c(1, 3, 2))
test_labs_ens <- aperm(abind(array(rep(aperm(test_labs, perm=c(2, 1, 3)), ens_n), c(r_n, samples_n, ens_n)), along=3), perm=c(1, 3, 2))
nets_test_cor_preds <- test_labs_nets == nets_test_top_indices
ens_tt_cor_preds <- test_labs_ens == ens_tt_top_indices
ens_vt_cor_preds <- test_labs_ens == ens_vt_top_indices
for (ri in 1:r_n)
{
  nets_cor_list <- list()
  incor <- 1:samples_n
  for (ni in 1:nets_n)
  {
    cor_list <- which(nets_test_cor_preds[ri, ni, ])
    nets_cor_list[[nets_outputs$networks[ni]]] = cor_list
    incor <- setdiff(incor, cor_list)
  }
  incor_n <- length(incor)
  
  for (ei in 1:ens_n)
  {
    cor_list_ens_tt <- which(ens_tt_cor_preds[ri, ei, ])
    cor_list_ens_vt <- which(ens_vt_cor_preds[ri, ei, ])
    
    cur_cor_list_tt <- nets_cor_list
    cur_cor_list_tt[[ens_outputs$methods[ei]]] = cor_list_ens_tt
    
    cur_cor_list_vt <- nets_cor_list
    cur_cor_list_vt[[ens_outputs$methods[ei]]] = cor_list_ens_vt
    
    
    venn_diag_tt <- ggVennDiagram(cur_cor_list_tt) + scale_fill_gradient(trans="log10", name="count", limits=c(1, 10000)) +
    ggtitle(paste("tt - replication ", ri)) +
      scale_x_continuous(limits=c(0, 1))
    
    venn_diag_vt <- ggVennDiagram(cur_cor_list_vt) + scale_fill_gradient(trans="log10", name="count", limits=c(1, 10000)) +
    ggtitle(paste("vt - replication ", ri)) +
      scale_x_continuous(limits=c(0, 1))
    
    print(venn_diag_tt)
    print(venn_diag_vt)
  }
}
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis
Warning: Transformation introduced infinite values in discrete y-axis

Ensemble in neither case managed to utilize all correct predictions shared amongs all pairs of networks.

LS0tDQp0aXRsZTogIlZpc3VhbGl6YXRpb25zIGNvcnJlY3Qgb3V0cHV0cyBhbW9uZ3MgbmV0d29ya3MgYW5kIGVuc2VtYmxlcyINCm91dHB1dDoNCiAgcGRmX2RvY3VtZW50OiBkZWZhdWx0DQogIGh0bWxfbm90ZWJvb2s6IGRlZmF1bHQNCi0tLQ0KDQpgYGB7cn0NCmxpYnJhcnkoZ2dwbG90MikNCmxpYnJhcnkoZHBseXIpDQpsaWJyYXJ5KHRpZHlyKQ0KbGlicmFyeSgiZ2dwdWJyIikNCmxpYnJhcnkoTERBVFMpDQpsaWJyYXJ5KGdnVmVubkRpYWdyYW0pDQpsaWJyYXJ5KHN0cmluZ3IpDQpsaWJyYXJ5KGFiaW5kKQ0KbGlicmFyeShwYXRjaHdvcmspDQoNCnNvdXJjZSgidXRpbHMuUiIpDQpgYGANCg0KVmlzdWFsaXphdGlvbiBvbiBuZXR3b3JrcyB0cmFpbmVkIGluIDMwIHJlcGxpY2F0aW9ucyBvbiBDSUZBUiAxMCB0cmFpbmluZyBzZXQgd2l0aCA1MDAgc2FtcGxlIHZhbGlkYXRpb24gc2V0IGV4dHJhY3RlZCByYW5kb21seSBpbiBlYWNoIHJlcGxpY2F0aW9uLg0KDQpgYGB7cn0NCmJhc2VfZGlyIDwtICIuLi9kYXRhL2RhdGFfdHJhaW5fdmFsX2MxMCINCnJlcGxzIDwtIDA6MjkNCmNsYXNzZXMgPC0gMTANCg0KbmV0c19vdXRwdXRzIDwtIGxvYWRfbmV0d29ya19vdXRwdXRzKGJhc2VfZGlyLCByZXBscykNCmVuc19vdXRwdXRzIDwtIGxvYWRfZW5zZW1ibGVfb3V0cHV0cyhiYXNlX2RpciwgcmVwbHMpDQpgYGANCg0KYGBge3J9DQpzb3J0X2luZCA8LSBmdW5jdGlvbihsc3QpDQp7DQogIHJldHVybihzb3J0KGxzdCwgaW5kZXgucmV0dXJuPVRSVUUsIGRlY3JlYXNpbmc9VFJVRSkkaXgpDQp9DQpuZXRzX3Rlc3RfdG9wX2luZGljZXMgPC0gYXBwbHkoWD1uZXRzX291dHB1dHMkdGVzdF9vdXRwdXRzLCBNQVJHSU49YygxLCAyLCAzKSwgRlVOPXNvcnRfaW5kKVsxLCAsICwgXQ0KZW5zX3R0X3RvcF9pbmRpY2VzIDwtIGFwcGx5KFg9ZW5zX291dHB1dHMkdHJhaW5fdHJhaW5pbmcsIE1BUkdJTj1jKDEsIDIsIDMpLCBGVU49c29ydF9pbmQpWzEsICwgLCBdDQplbnNfdnRfdG9wX2luZGljZXMgPC0gYXBwbHkoWD1lbnNfb3V0cHV0cyR2YWxfdHJhaW5pbmcsIE1BUkdJTj1jKDEsIDIsIDMpLCBGVU49c29ydF9pbmQpWzEsICwgLCBdDQpyX24gPC0gbGVuZ3RoKHJlcGxzKQ0Kc2FtcGxlc19uIDwtIGRpbShuZXRzX291dHB1dHMkdGVzdF9sYWJlbHMpWzJdDQpuZXRzX24gPC0gbGVuZ3RoKG5ldHNfb3V0cHV0cyRuZXR3b3JrcykNCmVuc19uIDwtIGxlbmd0aChlbnNfb3V0cHV0cyRtZXRob2RzKQ0KdGVzdF9sYWJzIDwtIG5ldHNfb3V0cHV0cyR0ZXN0X2xhYmVscyArIDENCmRpbSh0ZXN0X2xhYnMpIDwtIGMocl9uLCAxLCBzYW1wbGVzX24pDQp0ZXN0X2xhYnNfbmV0cyA8LSBhcGVybShhYmluZChhcnJheShyZXAoYXBlcm0odGVzdF9sYWJzLCBwZXJtPWMoMiwgMSwgMykpLCBuZXRzX24pLCBjKHJfbiwgc2FtcGxlc19uLCBuZXRzX24pKSwgYWxvbmc9MyksIHBlcm09YygxLCAzLCAyKSkNCnRlc3RfbGFic19lbnMgPC0gYXBlcm0oYWJpbmQoYXJyYXkocmVwKGFwZXJtKHRlc3RfbGFicywgcGVybT1jKDIsIDEsIDMpKSwgZW5zX24pLCBjKHJfbiwgc2FtcGxlc19uLCBlbnNfbikpLCBhbG9uZz0zKSwgcGVybT1jKDEsIDMsIDIpKQ0KbmV0c190ZXN0X2Nvcl9wcmVkcyA8LSB0ZXN0X2xhYnNfbmV0cyA9PSBuZXRzX3Rlc3RfdG9wX2luZGljZXMNCmVuc190dF9jb3JfcHJlZHMgPC0gdGVzdF9sYWJzX2VucyA9PSBlbnNfdHRfdG9wX2luZGljZXMNCmVuc192dF9jb3JfcHJlZHMgPC0gdGVzdF9sYWJzX2VucyA9PSBlbnNfdnRfdG9wX2luZGljZXMNCmBgYA0KDQpgYGB7cn0NCmZvciAocmkgaW4gMTpyX24pDQp7DQogIG5ldHNfY29yX2xpc3QgPC0gbGlzdCgpDQogIGluY29yIDwtIDE6c2FtcGxlc19uDQogIGZvciAobmkgaW4gMTpuZXRzX24pDQogIHsNCiAgICBjb3JfbGlzdCA8LSB3aGljaChuZXRzX3Rlc3RfY29yX3ByZWRzW3JpLCBuaSwgXSkNCiAgICBuZXRzX2Nvcl9saXN0W1tuZXRzX291dHB1dHMkbmV0d29ya3NbbmldXV0gPSBjb3JfbGlzdA0KICAgIGluY29yIDwtIHNldGRpZmYoaW5jb3IsIGNvcl9saXN0KQ0KICB9DQogIGluY29yX24gPC0gbGVuZ3RoKGluY29yKQ0KICANCiAgZm9yIChlaSBpbiAxOmVuc19uKQ0KICB7DQogICAgY29yX2xpc3RfZW5zX3R0IDwtIHdoaWNoKGVuc190dF9jb3JfcHJlZHNbcmksIGVpLCBdKQ0KICAgIGNvcl9saXN0X2Vuc192dCA8LSB3aGljaChlbnNfdnRfY29yX3ByZWRzW3JpLCBlaSwgXSkNCiAgICANCiAgICBjdXJfY29yX2xpc3RfdHQgPC0gbmV0c19jb3JfbGlzdA0KICAgIGN1cl9jb3JfbGlzdF90dFtbZW5zX291dHB1dHMkbWV0aG9kc1tlaV1dXSA9IGNvcl9saXN0X2Vuc190dA0KICAgIA0KICAgIGN1cl9jb3JfbGlzdF92dCA8LSBuZXRzX2Nvcl9saXN0DQogICAgY3VyX2Nvcl9saXN0X3Z0W1tlbnNfb3V0cHV0cyRtZXRob2RzW2VpXV1dID0gY29yX2xpc3RfZW5zX3Z0DQogICAgDQogICAgDQogICAgdmVubl9kaWFnX3R0IDwtIGdnVmVubkRpYWdyYW0oY3VyX2Nvcl9saXN0X3R0KSArIHNjYWxlX2ZpbGxfZ3JhZGllbnQodHJhbnM9ImxvZzEwIiwgbmFtZT0iY291bnQiLCBsaW1pdHM9YygxLCAxMDAwMCkpICsNCiAgICBnZ3RpdGxlKHBhc3RlKCJ0dCAtIHJlcGxpY2F0aW9uICIsIHJpKSkgKw0KICAgICAgc2NhbGVfeF9jb250aW51b3VzKGxpbWl0cz1jKDAsIDEpKQ0KICAgIA0KICAgIHZlbm5fZGlhZ192dCA8LSBnZ1Zlbm5EaWFncmFtKGN1cl9jb3JfbGlzdF92dCkgKyBzY2FsZV9maWxsX2dyYWRpZW50KHRyYW5zPSJsb2cxMCIsIG5hbWU9ImNvdW50IiwgbGltaXRzPWMoMSwgMTAwMDApKSArDQogICAgZ2d0aXRsZShwYXN0ZSgidnQgLSByZXBsaWNhdGlvbiAiLCByaSkpICsNCiAgICAgIHNjYWxlX3hfY29udGludW91cyhsaW1pdHM9YygwLCAxKSkNCiAgICANCiAgICBwcmludCh2ZW5uX2RpYWdfdHQpDQogICAgcHJpbnQodmVubl9kaWFnX3Z0KQ0KICB9DQp9DQpgYGANCkVuc2VtYmxlIGluIG5laXRoZXIgY2FzZSBtYW5hZ2VkIHRvIHV0aWxpemUgYWxsIGNvcnJlY3QgcHJlZGljdGlvbnMgc2hhcmVkIGFtb25ncyBhbGwgcGFpcnMgb2YgbmV0d29ya3MuDQoNCg0K